#coding:utf-8
"""
*   TG_Coder : Dzlua
*   Email    : 505544956@qq.com
*   Time     : 2017/08/28
"""

from database import DataBase
from bs4 import BeautifulSoup
import re
import codecs

class Merge(DataBase):
    def __init__(self, host, user, pwd, db):
        self.__tb = 'librarys';
        DataBase.__init__(self, host, user, pwd, db)
        sql = "CREATE TABLE IF NOT EXISTS " + self.__tb + " " \
            "(id INT AUTO_INCREMENT PRIMARY KEY," \
            " name VARCHAR(64) NOT NULL," \
            " star INT NOT NULL DEFAULT 0," \
            " type VARCHAR(256) NOT NULL DEFAULT ''," \
            " lang VARCHAR(64) NOT NULL DEFAULT ''," \
            " info_text TEXT," \
            " info TEXT," \
            " home TEXT," \
            " os VARCHAR(64) NOT NULL DEFAULT ''," \
            " license VARCHAR(64) NOT NULL DEFAULT ''," \
            " local_path TEXT," \
            " add_date TIMESTAMP," \
            " INDEX(name, star), UNIQUE(name) )" \
            " DEFAULT CHARSET=utf8;"
        DataBase.execute(self, sql)

        self.read(self.__on_read)
    def write(self, text):
        f = codecs.open(self.__tb + '.log', 'a', 'utf-8')
        f.write(text)
        f.close()
    def _select(self, conn, sql):
        cursor = conn.cursor()
        cursor.execute(sql)
        result = cursor.fetchall()
        cursor.close()
        return result
    def __on_read(self, conn, args):
        sql = "SELECT * FROM libdownload WHERE download = 3;"
        result = self._select(conn, sql)
        length = len(result)
        for r in result:
            print('[Merge] id:%s/%d' % (r[0], length))
            info = {'name': '', 'star': 0, 'type': r[7], 'lang': r[4],
                    'info_text': '', 'info': '', 'home': '', 'os': '',
                    'license': '', 'local_path': r[7], 'id': r[0] }
            if not self.__parser(conn, info, r):
                continue
            sql = "SELECT id FROM %s WHERE name='%s'" % (self.__tb, info['name'])
            result = self._select(conn, sql)
            if result:
                continue
            self.save(self.__on_save, info)
    def __parser(self, conn, info, data):
        # star
        sql = "SELECT star FROM libstar WHERE name = '%s';" % (data[1])
        result = self._select(conn, sql)
        for r in result:
            info['star'] = r[0]
            break
        # name,info,info_text,oslicense
        sql = "SELECT project,info,os,license,name FROM libinfo WHERE href = '%s';" % (data[1])
        result = self._select(conn, sql)
        for r in result:
            info['name'] = r[0]
            info['info'] = r[1]
            info['os'] = r[2]
            info['license'] = r[3]
            info['home'] = r[4]
            info['info_text'] = self.__parser_text(r[1])
            break
        if info['name'] is '':
            return False
        return True
    def __parser_text(self, xml):
        soup = BeautifulSoup(xml, "lxml")
        div = soup.find('div')
        if not div:
            print('[Merge] not div.')
            return
        text = div.get_text()
        text = re.sub(r'\n{2,}', '', text)
        return text
    def __on_save(self, conn, data, args):
        try:
            cursor = conn.cursor()
            sql = "INSERT INTO " + self.__tb + " (name,star,type,lang,info_text,info,home,os,license,local_path)" \
                " VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);"
            param = ( data['name'], data['star'], data['type'], data['lang'], data['info_text'], \
                    data['info'], data['home'], data['os'],data['license'],data['local_path'] )
            cursor.execute(sql, param)
            cursor.close()
            conn.commit()
        except conn.Error, e:
            err = '[Merge] Error __on_save : %d %s, %s ' % (e[0], e[1], data)
            self.write(err + '\n')
            print(err)
        finally:
            print('[Merge] save id: %d, name: %s' % (data['id'], data['name']))
#------------
merge = Merge('localhost', 'dzlua', 'dzlua', 'spider')
merge.close()